#!/bin/bash

# ============================================
# Prometheus监控系统验证脚本 (Prometheus Monitoring System Verification Script)
# ============================================
# 功能：全面验证Prometheus监控系统的部署和功能
# Function: Comprehensive verification of Prometheus monitoring system deployment and functionality
# 作者：系统管理员 (Author: System Administrator)
# 版本：1.0 (Version: 1.0)
# 日期：2024-01-01 (Date: 2024-01-01)
# 用法：./verify-monitoring.sh (Usage: ./verify-monitoring.sh)
#
# 详细功能说明 (Detailed Function Description):
# 1. 检查Prometheus服务状态 (Check Prometheus service status)
#    - 验证systemd服务运行状态 (Verify systemd service running status)
#    - 检查进程是否正常运行 (Check if process is running normally)
#    - 验证服务启动配置 (Verify service startup configuration)
#
# 2. 验证Prometheus配置文件 (Verify Prometheus configuration files)
#    - 检查主配置文件语法 (Check main config file syntax)
#    - 验证告警规则文件语法 (Verify alert rules file syntax)
#    - 检查配置文件权限 (Check config file permissions)
#
# 3. 检查监控目标状态 (Check monitoring target status)
#    - 验证目标主机连通性 (Verify target host connectivity)
#    - 检查指标收集状态 (Check metrics collection status)
#    - 验证服务发现配置 (Verify service discovery configuration)
#
# 4. 验证告警规则 (Verify alert rules)
#    - 检查告警规则加载状态 (Check alert rules loading status)
#    - 验证告警规则语法 (Verify alert rules syntax)
#    - 测试告警触发机制 (Test alert triggering mechanism)
#
# 5. 测试Web界面访问 (Test web interface access)
#    - 验证HTTP端口监听 (Verify HTTP port listening)
#    - 测试API接口响应 (Test API interface response)
#    - 检查Web界面功能 (Check web interface functionality)
#
# 6. 生成监控系统报告 (Generate monitoring system report)
#    - 汇总验证结果 (Summarize verification results)
#    - 提供配置建议 (Provide configuration recommendations)
#    - 输出系统状态摘要 (Output system status summary)
#
# ============================================
# 颜色配置 (Color Configuration)
# ============================================
# 用于美化输出显示，定义各种颜色代码，提升脚本输出的可读性和用户体验
# Used to beautify output display, define various color codes to improve readability and user experience
# 使用ANSI转义序列实现终端颜色显示 (Use ANSI escape sequences for terminal color display)
# ============================================
RED='\033[0;31m'      # 红色 - 用于显示错误信息和失败状态 (Red - for error messages and failure status)
GREEN='\033[0;32m'    # 绿色 - 用于显示成功信息和正常状态 (Green - for success messages and normal status)
YELLOW='\033[1;33m'   # 黄色 - 用于显示警告信息和需要注意的项目 (Yellow - for warning messages and items requiring attention)
BLUE='\033[0;34m'     # 蓝色 - 用于显示标题、分组和一般信息 (Blue - for titles, grouping and general information)
NC='\033[0m'          # 无颜色 - 重置颜色设置，恢复终端默认颜色 (No Color - reset color settings, restore terminal default color)

# ============================================
# 脚本标题显示
# 显示脚本名称、执行时间和装饰性边框
# 提供清晰的脚本执行开始标识
# ============================================
echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║                  Prometheus监控验证                 ║${NC}"
echo -e "${BLUE}║                     $(date '+%Y-%m-%d %H:%M:%S')                     ║${NC}"
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
echo

# ============================================
# 1. 检查Prometheus服务状态 (Check Prometheus Service Status)
# ============================================
# 功能说明 (Function Description):
#   验证Prometheus监控服务是否正常启动和运行，确保所有监控节点的服务状态一致
#   Verify Prometheus monitoring service starts and runs normally, ensure consistent service status across all monitoring nodes
# 检查项目 (Check Items):
#   - 使用Ansible systemd模块验证服务运行状态 (Use Ansible systemd module to verify service running status)
#   - 确认服务是否已启动并处于活动状态 (Confirm service is started and in active state)
#   - 检查所有监控节点上的服务状态一致性 (Check service status consistency across all monitoring nodes)
#   - 验证systemd服务的启用状态和自启动配置 (Verify systemd service enabled status and auto-start configuration)
# 预期结果 (Expected Results):
#   所有节点上的Prometheus服务都应该处于started状态，返回SUCCESS状态码
#   All Prometheus services on all nodes should be in started state, returning SUCCESS status code
# 故障排查 (Troubleshooting):
#   如果服务未启动，检查配置文件语法、端口占用、权限设置等问题
#   If service is not started, check config file syntax, port occupation, permission settings, etc.
# ============================================
echo -e "${BLUE}1. 检查Prometheus服务状态... (Checking Prometheus Service Status...)${NC}"
echo -e "   ${YELLOW}→ 验证所有监控节点上的Prometheus服务运行状态 (Verifying Prometheus service running status on all monitoring nodes)${NC}"
echo -e "   ${BLUE}→ 使用Ansible systemd模块进行批量状态检查 (Using Ansible systemd module for batch status checking)${NC}"
ansible monitoring -i inventories/production -m systemd -a "name=prometheus state=started" --one-line
echo

# ============================================
# 2. 检查Prometheus端口监听状态 (Check Prometheus Port Listening Status)
# ============================================
# 功能说明 (Function Description):
#   验证Prometheus服务是否在正确的端口上监听，确保网络连接正常
#   Verify Prometheus service is listening on correct port, ensure network connection is normal
# 检查项目 (Check Items):
#   - 使用ss命令检查TCP端口9090的监听状态 (Use ss command to check TCP port 9090 listening status)
#   - 验证Prometheus进程是否正确绑定到网络端口 (Verify Prometheus process correctly binds to network port)
#   - 确认网络服务的可用性和进程关联 (Confirm network service availability and process association)
#   - 检查端口绑定的IP地址范围和访问权限 (Check port binding IP address range and access permissions)
# 预期结果 (Expected Results):
#   应该显示Prometheus进程在0.0.0.0:9090或:::9090上监听，状态为LISTEN
#   Should show Prometheus process listening on 0.0.0.0:9090 or :::9090, status should be LISTEN
# 网络配置 (Network Configuration):
#   确保防火墙规则允许9090端口的访问，配置适当的网络安全策略
#   Ensure firewall rules allow access to port 9090, configure appropriate network security policies
# 故障排查 (Troubleshooting):
#   如果端口未监听，检查配置文件中的web.listen-address参数设置
#   If port is not listening, check web.listen-address parameter setting in config file
# ============================================
echo -e "${BLUE}2. 检查Prometheus端口监听状态... (Checking Prometheus Port Listening Status...)${NC}"
echo -e "   ${YELLOW}→ 验证Prometheus在端口9090上的网络监听状态 (Verifying Prometheus network listening status on port 9090)${NC}"
echo -e "   ${BLUE}→ 使用ss命令检查TCP端口状态和进程绑定 (Using ss command to check TCP port status and process binding)${NC}"
ansible monitoring -i inventories/production -m shell -a "ss -tlnp | grep :9090" --one-line
echo

# ============================================
# 3. 检查Web界面健康状态 (Check Web Interface Health Status)
# ============================================
# 功能说明 (Function Description):
#   验证Prometheus Web界面是否可访问且健康，确保监控系统的Web服务正常
#   Verify Prometheus web interface is accessible and healthy, ensure monitoring system web service is normal
# 检查项目 (Check Items):
#   - 通过HTTP GET请求访问/-/healthy健康检查端点 (Access /-/healthy health check endpoint via HTTP GET request)
#   - 验证Web服务的HTTP响应状态码（期望200） (Verify web service HTTP response status code, expect 200)
#   - 确认Prometheus内部组件的健康状态 (Confirm health status of Prometheus internal components)
#   - 测试Web UI的基本可访问性 (Test basic accessibility of Web UI)
# 预期结果 (Expected Results):
#   返回HTTP 200状态码，表示Prometheus服务健康
#   Return HTTP 200 status code, indicating Prometheus service is healthy
# 健康检查意义 (Health Check Significance):
#   健康检查端点用于监控系统自身的状态，是运维监控的重要指标
#   Health check endpoint is used to monitor system's own status, important indicator for operations monitoring
# 故障排查 (Troubleshooting):
#   如果健康检查失败，可能是配置错误、资源不足或依赖服务异常
#   If health check fails, may be due to configuration errors, insufficient resources or dependent service anomalies
# ============================================
echo -e "${BLUE}3. 检查Web界面健康状态... (Checking Web Interface Health Status...)${NC}"
echo -e "   ${YELLOW}→ 通过健康检查端点验证Web界面可访问性 (Verifying web interface accessibility via health check endpoint)${NC}"
echo -e "   ${BLUE}→ 使用Ansible uri模块进行HTTP健康检查 (Using Ansible uri module for HTTP health check)${NC}"
ansible monitoring -i inventories/production -m uri -a "url=http://localhost:9090/-/healthy method=GET status_code=200" --one-line
echo

# ============================================
# 4. 验证配置文件语法 (Verify Configuration File Syntax)
# ============================================
# 功能说明 (Function Description):
#   使用promtool工具验证Prometheus配置文件语法，确保配置变更不会导致服务启动失败
#   Use promtool to verify Prometheus configuration file syntax, ensure configuration changes won't cause service startup failure
# 检查项目 (Check Items):
#   - 使用官方promtool工具检查主配置文件语法 (Use official promtool to check main configuration file syntax)
#   - 验证YAML格式的正确性和配置项的有效性 (Verify YAML format correctness and configuration item validity)
#   - 检查scrape_configs、rule_files等关键配置段 (Check key configuration sections like scrape_configs, rule_files)
#   - 确保配置文件可以被Prometheus正确解析和加载 (Ensure configuration file can be correctly parsed and loaded by Prometheus)
# 配置文件重要性 (Configuration File Importance):
#   配置文件是Prometheus运行的核心，语法错误会导致服务无法启动
#   Configuration file is core of Prometheus operation, syntax errors will cause service startup failure
# 预期结果 (Expected Results):
#   promtool应该报告"SUCCESS: prometheus.yml is valid"，无语法错误
#   promtool should report "SUCCESS: prometheus.yml is valid", no syntax errors
# 故障排查 (Troubleshooting):
#   如果语法检查失败，检查YAML缩进、引号使用、配置项拼写等
#   If syntax check fails, check YAML indentation, quote usage, configuration item spelling, etc.
# ============================================
echo -e "${BLUE}4. 验证配置文件语法... (Verifying Configuration File Syntax...)${NC}"
echo -e "   ${YELLOW}→ 使用promtool检查主配置文件的语法正确性 (Using promtool to check main configuration file syntax correctness)${NC}"
echo -e "   ${BLUE}→ 验证/etc/prometheus/prometheus.yml配置文件 (Verifying /etc/prometheus/prometheus.yml configuration file)${NC}"
ansible monitoring -i inventories/production -m shell -a "/opt/prometheus/promtool check config /etc/prometheus/prometheus.yml" --one-line
echo

# ============================================
# 5. 验证告警规则语法 (Verify Alert Rules Syntax)
# ============================================
# 功能说明 (Function Description):
#   使用promtool工具验证告警规则文件语法，确保告警系统能够正常工作
#   Use promtool to verify alert rules file syntax, ensure alert system can work normally
# 检查项目 (Check Items):
#   - 使用promtool检查告警规则文件的语法正确性 (Use promtool to check alert rules file syntax correctness)
#   - 验证PromQL表达式的有效性和语法 (Verify PromQL expression validity and syntax)
#   - 检查告警规则的结构和必需字段 (Check alert rules structure and required fields)
#   - 确保告警规则可以被Prometheus正确加载和执行 (Ensure alert rules can be correctly loaded and executed by Prometheus)
#   - 验证告警标签和注释的格式规范性 (Verify format compliance of alert labels and annotations)
# 告警规则重要性 (Alert Rules Importance):
#   告警规则是监控系统的核心功能，语法错误会导致告警失效和监控盲区
#   Alert rules are core functionality of monitoring system, syntax errors will cause alert failure and monitoring blind spots
# 预期结果 (Expected Results):
#   promtool应该报告规则文件语法正确，无错误信息，返回"SUCCESS"状态
#   promtool should report rules file syntax is correct, no error messages, return "SUCCESS" status
# 规则文件结构 (Rules File Structure):
#   包含groups、rules、alert、expr、for、labels、annotations等关键字段
#   Contains key fields like groups, rules, alert, expr, for, labels, annotations
# 故障排查 (Troubleshooting):
#   如果规则检查失败，检查PromQL语法、YAML格式、字段拼写、缩进等
#   If rules check fails, check PromQL syntax, YAML format, field spelling, indentation, etc.
# ============================================
echo -e "${BLUE}5. 验证告警规则语法... (Verifying Alert Rules Syntax...)${NC}"
echo -e "   ${YELLOW}→ 使用promtool检查告警规则文件的语法和PromQL表达式 (Using promtool to check alert rules file syntax and PromQL expressions)${NC}"
echo -e "   ${BLUE}→ 验证/etc/prometheus/rules/alert_rules.yml规则文件 (Verifying /etc/prometheus/rules/alert_rules.yml rules file)${NC}"
ansible monitoring -i inventories/production -m shell -a "/opt/prometheus/promtool check rules /etc/prometheus/rules/alert_rules.yml" --one-line
echo

# ============================================
# 6. 检查监控目标状态 (Check Monitoring Target Status)
# ============================================
# 功能说明 (Function Description):
#   验证Prometheus监控目标的连接和数据采集状态，确保所有目标正常工作
#   Verify Prometheus monitoring target connection and data collection status, ensure all targets work normally
# 检查项目 (Check Items):
#   - 通过Prometheus API获取所有配置的监控目标状态 (Get all configured monitoring target status via Prometheus API)
#   - 验证各个target的健康状态（up/down） (Verify health status of each target - up/down)
#   - 检查数据采集的连通性和最后采集时间 (Check data collection connectivity and last scrape time)
#   - 确认所有配置的job和instance都能正常工作 (Confirm all configured jobs and instances work normally)
#   - 检查目标的标签和元数据信息 (Check target labels and metadata information)
# 监控目标重要性 (Monitoring Target Importance):
#   监控目标是数据来源，目标异常会导致监控数据缺失和告警失效
#   Monitoring targets are data sources, target anomalies will cause monitoring data loss and alert failure
# 预期结果 (Expected Results):
#   API返回成功，所有重要目标状态为"up"，能够正常采集指标数据
#   API returns success, all important targets status should be "up", able to collect metrics data normally
# API端点说明 (API Endpoint Description):
#   /api/v1/targets端点返回所有目标的详细状态信息，包括健康状态、错误信息等
#   /api/v1/targets endpoint returns detailed status information of all targets, including health status, error messages, etc.
# 故障排查 (Troubleshooting):
#   如果目标状态为DOWN，检查网络连通性、目标服务状态、防火墙配置等
#   If target status is DOWN, check network connectivity, target service status, firewall configuration, etc.
# ============================================
echo -e "${BLUE}6. 检查监控目标状态... (Checking Monitoring Target Status...)${NC}"
echo -e "   ${YELLOW}→ 通过API查询所有监控目标的连接和采集状态 (Querying all monitoring target connection and collection status via API)${NC}"
echo -e "   ${BLUE}→ 使用Prometheus API /api/v1/targets端点 (Using Prometheus API /api/v1/targets endpoint)${NC}"
ansible monitoring -i inventories/production -m uri -a "url=http://localhost:9090/api/v1/targets method=GET" --one-line
echo

# ============================================
# 7. 检查配置加载状态 (Check Configuration Load Status)
# ============================================
# 功能说明 (Function Description):
#   验证Prometheus配置的加载状态和时间，确保配置变更已正确应用
#   Verify Prometheus configuration load status and time, ensure configuration changes are correctly applied
# 检查项目 (Check Items):
#   - 通过status/config API端点获取当前生效的配置 (Get current effective configuration via status/config API endpoint)
#   - 验证配置文件是否成功加载到内存中 (Verify configuration file is successfully loaded into memory)
#   - 检查配置的最后加载时间和版本信息 (Check configuration last load time and version information)
#   - 确认运行时配置与文件系统中的配置一致性 (Confirm runtime configuration consistency with filesystem configuration)
# 配置加载重要性 (Configuration Load Importance):
#   配置加载状态反映了系统的当前运行配置，确保配置变更生效
#   Configuration load status reflects system's current running configuration, ensures configuration changes take effect
# 预期结果 (Expected Results):
#   API返回HTTP 200，显示当前加载的完整配置内容，无错误信息
#   API returns HTTP 200, displays current loaded complete configuration content, no error messages
# API端点说明 (API Endpoint Description):
#   /api/v1/status/config端点返回当前加载的配置信息和状态
#   /api/v1/status/config endpoint returns currently loaded configuration information and status
# 配置重载机制 (Configuration Reload Mechanism):
#   Prometheus支持热重载配置，无需重启服务即可应用新配置
#   Prometheus supports hot reload configuration, can apply new configuration without restarting service
# 故障排查 (Troubleshooting):
#   如果配置加载失败，检查配置文件语法、权限、文件路径等
#   If configuration load fails, check config file syntax, permissions, file path, etc.
# ============================================
echo -e "${BLUE}7. 检查配置加载状态... (Checking Configuration Load Status...)${NC}"
echo -e "   ${YELLOW}→ 验证当前运行时配置的加载状态和内容 (Verifying current runtime configuration load status and content)${NC}"
echo -e "   ${BLUE}→ 使用Prometheus API /api/v1/status/config端点 (Using Prometheus API /api/v1/status/config endpoint)${NC}"
ansible monitoring -i inventories/production -m uri -a "url=http://localhost:9090/api/v1/status/config method=GET status_code=200" --one-line
echo

# ============================================
# 8. 检查告警规则状态 (Check Alert Rules Status)
# ============================================
# 功能说明 (Function Description):
#   验证告警规则的加载和执行状态，确保告警系统能够正常监控和报警
#   Verify alert rules loading and execution status, ensure alert system can monitor and alert normally
# 检查项目 (Check Items):
#   - 通过rules API端点获取所有加载的告警规则 (Get all loaded alert rules via rules API endpoint)
#   - 验证告警规则的加载状态和执行情况 (Verify alert rules loading status and execution status)
#   - 检查规则的评估时间和当前状态（pending/firing/inactive） (Check rules evaluation time and current status - pending/firing/inactive)
#   - 确认告警规则组的配置和规则数量 (Confirm alert rule groups configuration and rules count)
#   - 验证PromQL表达式的执行结果和性能 (Verify PromQL expression execution results and performance)
# 告警规则重要性 (Alert Rules Importance):
#   告警规则是监控系统的核心功能，确保及时发现和响应系统异常
#   Alert rules are core functionality of monitoring system, ensure timely detection and response to system anomalies
# 预期结果 (Expected Results):
#   API返回HTTP 200，显示所有告警规则的详细状态信息，规则状态正常
#   API returns HTTP 200, displays detailed status information of all alert rules, rules status should be normal
# API端点说明 (API Endpoint Description):
#   /api/v1/rules端点返回所有规则组和规则的详细状态信息，包括评估时间、状态等
#   /api/v1/rules endpoint returns detailed status information of all rule groups and rules, including evaluation time, status, etc.
# 规则评估机制 (Rules Evaluation Mechanism):
#   Prometheus定期评估告警规则，根据PromQL表达式判断是否触发告警
#   Prometheus periodically evaluates alert rules, determines whether to trigger alerts based on PromQL expressions
# 故障排查 (Troubleshooting):
#   如果规则状态异常，检查PromQL语法、数据可用性、评估间隔等
#   If rules status is abnormal, check PromQL syntax, data availability, evaluation interval, etc.
# ============================================
echo -e "${BLUE}8. 检查告警规则状态... (Checking Alert Rules Status...)${NC}"
echo -e "   ${YELLOW}→ 验证告警规则的加载状态和当前执行情况 (Verifying alert rules loading status and current execution status)${NC}"
echo -e "   ${BLUE}→ 使用Prometheus API /api/v1/rules端点 (Using Prometheus API /api/v1/rules endpoint)${NC}"
ansible monitoring -i inventories/production -m uri -a "url=http://localhost:9090/api/v1/rules method=GET status_code=200" --one-line
echo

# ============================================
# 验证完成总结 (Verification Completion Summary)
# ============================================
# 功能说明 (Function Description):
#   显示验证过程的完成状态和总体结果，确认Prometheus监控系统的部署状态
#   Display verification process completion status and overall results, confirm Prometheus monitoring system deployment status
# 输出内容 (Output Content):
#   - 验证完成的确认信息 (Verification completion confirmation information)
#   - 部署成功的判断标准说明 (Deployment success criteria description)
#   - 后续操作的指导建议 (Follow-up operation guidance recommendations)
#   - 系统就绪状态确认 (System readiness status confirmation)
# 验证结果意义 (Verification Results Significance):
#   通过8个关键检查项目的验证，确保监控系统的完整性和可用性
#   Through verification of 8 key check items, ensure monitoring system integrity and availability
# 部署成功标准 (Deployment Success Criteria):
#   所有检查项目通过表示Prometheus监控系统已成功部署并可正常使用
#   All check items passed indicates Prometheus monitoring system has been successfully deployed and ready for use
# ============================================
echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║                        验证完成                              ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}"
echo
echo -e "${GREEN}✅ 部署验证结果：${NC}"
echo -e "   ${YELLOW}•${NC} Prometheus服务已启动并运行正常"
echo -e "   ${YELLOW}•${NC} Web界面可正常访问"
echo -e "   ${YELLOW}•${NC} 配置文件语法正确"
echo -e "   ${YELLOW}•${NC} 告警规则配置有效"
echo -e "   ${BLUE}→ 监控系统已准备就绪，可以开始收集和分析指标数据${NC}"
echo

# ============================================
# Web界面访问地址 (Web Interface Access Addresses)
# ============================================
# 功能说明 (Function Description):
#   提供Prometheus Web界面的访问方式和地址信息，包含各服务器的详细访问信息
#   Provide Prometheus Web interface access methods and address information, including detailed access info for each server
# 输出内容 (Output Content):
#   - 各服务器的访问地址（IP地址和端口） (Access addresses for each server - IP addresses and ports)
#   - 访问权限和安全注意事项 (Access permissions and security considerations)
#   - 网络配置要求说明 (Network configuration requirements description)
#   - 服务器角色和用途说明 (Server roles and usage description)
# 服务器部署架构 (Server Deployment Architecture):
#   采用分布式部署架构，不同服务器承担不同的监控职责
#   Uses distributed deployment architecture, different servers handle different monitoring responsibilities
# 网络访问要求 (Network Access Requirements):
#   确保客户端能够访问各服务器的9090端口，网络连通性正常
#   Ensure clients can access port 9090 on each server, network connectivity is normal
# 安全配置建议 (Security Configuration Recommendations):
#   生产环境建议配置防火墙规则、访问控制列表和身份验证机制
#   Production environment recommends configuring firewall rules, access control lists and authentication mechanisms
# ============================================
echo -e "${BLUE}🌐 Web界面访问地址 (Web Interface Access Addresses)：${NC}"
echo -e "   ${YELLOW}•${NC} app-server-01: ${GREEN}http://192.168.32.41:9090${NC}"
echo -e "     ${BLUE}→ 应用服务器监控节点，适用于应用层指标查看 (Application server monitoring node, suitable for application-level metrics viewing)${NC}"
echo -e "   ${YELLOW}•${NC} app-server-02: ${GREEN}http://192.168.32.42:9090${NC}"
echo -e "     ${BLUE}→ 应用服务器监控节点，提供负载均衡和高可用 (Application server monitoring node, provides load balancing and high availability)${NC}"
echo -e "   ${YELLOW}•${NC} db-server: ${GREEN}http://192.168.32.46:9090${NC}"
echo -e "     ${BLUE}→ 数据库服务器监控节点，专注于数据库性能指标 (Database server monitoring node, focuses on database performance metrics)${NC}"
echo -e "   ${YELLOW}安全提醒 (Security Reminders)：${NC}"
echo -e "     ${YELLOW}→ 请确保防火墙已正确配置9090端口访问规则 (Please ensure firewall has correctly configured port 9090 access rules)${NC}"
echo -e "     ${YELLOW}→ 建议配置访问控制和身份验证机制 (Recommend configuring access control and authentication mechanisms)${NC}"
echo

# ============================================
# 关键Web UI页面说明 (Key Web UI Pages Description)
# ============================================
# 功能说明 (Function Description):
#   详细介绍Prometheus Web界面的核心功能页面和使用方法
#   Detailed introduction to core functional pages and usage methods of Prometheus Web interface
# 页面功能概述 (Page Function Overview):
#   每个页面都有特定的监控和管理功能，帮助用户全面了解系统状态
#   Each page has specific monitoring and management functions, helping users comprehensively understand system status
# 使用场景说明 (Usage Scenario Description):
#   - Graph页面：日常监控和故障排查的主要工具 (Graph page: Main tool for daily monitoring and troubleshooting)
#   - Alerts页面：告警管理和响应的核心界面 (Alerts page: Core interface for alert management and response)
#   - Targets页面：监控目标健康状态检查 (Targets page: Monitoring target health status check)
#   - Config页面：配置验证和问题诊断 (Config page: Configuration verification and problem diagnosis)
# 操作建议 (Operation Recommendations):
#   建议定期检查各页面状态，确保监控系统正常运行
#   Recommend regularly checking status of each page to ensure monitoring system runs normally
# ============================================
echo -e "${BLUE}📊 主要功能页面 (Key Web UI Pages)：${NC}"
echo -e "   ${YELLOW}•${NC} Graph页面 (Graph Page): ${GREEN}/graph${NC} (PromQL查询和图表)"
echo -e "     ${BLUE}→ 使用PromQL查询语言进行指标查询和图表绘制 (Use PromQL query language for metrics querying and chart plotting)${NC}"
echo -e "     ${BLUE}→ 支持复杂查询表达式和图表展示 (Supports complex query expressions and chart display)${NC}"
echo -e "   ${YELLOW}•${NC} Alerts页面 (Alerts Page): ${GREEN}/alerts${NC} (告警规则状态)"
echo -e "     ${BLUE}→ 查看告警规则状态、活跃告警和历史告警信息 (View alert rules status, active alerts and historical alert information)${NC}"
echo -e "     ${BLUE}→ 告警历史记录和触发条件详情 (Alert history records and trigger condition details)${NC}"
echo -e "   ${YELLOW}•${NC} Targets页面 (Targets Page): ${GREEN}/targets${NC} (监控目标状态)"
echo -e "     ${BLUE}→ 监控所有配置的采集目标状态和连接健康度 (Monitor all configured collection target status and connection health)${NC}"
echo -e "     ${BLUE}→ 显示目标健康状态和最后抓取时间 (Display target health status and last scrape time)${NC}"
echo -e "   ${YELLOW}•${NC} Configuration页面 (Configuration Page): ${GREEN}/config${NC} (配置文件查看)"
echo -e "     ${BLUE}→ 查看当前生效的Prometheus配置文件内容 (View current effective Prometheus configuration file content)${NC}"
echo -e "     ${BLUE}→ 配置语法高亮和结构化显示 (Configuration syntax highlighting and structured display)${NC}"
echo -e "   ${YELLOW}•${NC} Service Discovery: ${GREEN}/service-discovery${NC} (服务发现)"
echo -e "     ${BLUE}→ 服务发现状态，动态目标发现和注册情况 (Service discovery status, dynamic target discovery and registration status)${NC}"
echo -e "     ${BLUE}→ 自动发现的服务实例和标签信息 (Automatically discovered service instances and label information)${NC}"
echo

# ============================================
# 常用管理命令 (Common Management Commands)
# ============================================
# 功能说明 (Function Description):
#   提供Prometheus监控系统的日常管理和维护命令，涵盖配置管理、服务控制、日志查看等核心操作
#   Provide daily management and maintenance commands for Prometheus monitoring system, covering configuration management, service control, log viewing and other core operations
# 命令分类 (Command Categories):
#   - 配置管理：重载和验证配置文件，确保配置变更正确应用 (Configuration management: reload and verify config files, ensure configuration changes are correctly applied)
#   - 服务管理：启动、停止、重启服务，控制服务生命周期 (Service management: start, stop, restart services, control service lifecycle)
#   - 日志查看：实时监控和故障排查，获取运行状态信息 (Log viewing: real-time monitoring and troubleshooting, get runtime status information)
#   - 状态检查：服务状态和健康监控，确保系统正常运行 (Status checking: service status and health monitoring, ensure system runs normally)
# 使用场景 (Usage Scenarios):
#   配置更新后需要重新加载，故障发生时需要查看日志，性能调优时需要重启服务
#   Need to reload after configuration updates, need to view logs when failures occur, need to restart service for performance tuning
# 安全注意事项 (Security Considerations):
#   管理命令需要适当的系统权限，建议使用sudo执行，避免权限不足导致操作失败
#   Management commands require appropriate system permissions, recommend using sudo, avoid operation failure due to insufficient permissions
# 最佳实践 (Best Practices):
#   配置变更前先备份，重启前先检查语法，操作后验证结果
#   Backup before configuration changes, check syntax before restart, verify results after operations
# ============================================
echo -e "${BLUE}🔧 常用管理命令 (Common Management Commands)：${NC}"
echo -e "   ${YELLOW}•${NC} 重载配置 (Reload Configuration): ${GREEN}curl -X POST http://localhost:9090/-/reload${NC}"
echo -e "     ${BLUE}→ 热重载配置文件，无需重启服务即可应用新配置 (Hot reload configuration files, apply new configuration without restarting service)${NC}"
echo -e "     ${BLUE}→ 推荐用于配置文件更新后的快速生效 (Recommended for quick effect after configuration file updates)${NC}"
echo -e "   ${YELLOW}•${NC} 检查配置 (Check Configuration): ${GREEN}/opt/prometheus/promtool check config /etc/prometheus/prometheus.yml${NC}"
echo -e "     ${BLUE}→ 验证配置文件语法正确性，避免配置错误导致服务异常 (Verify configuration file syntax correctness, avoid service anomalies caused by configuration errors)${NC}"
echo -e "     ${BLUE}→ 建议在重新加载前先执行语法检查 (Recommend running syntax check before reload)${NC}"
echo -e "   ${YELLOW}•${NC} 查看日志 (View Logs): ${GREEN}sudo journalctl -u prometheus -f${NC}"
echo -e "     ${BLUE}→ 实时查看服务日志，用于故障排查和状态监控 (Real-time view of service logs, used for troubleshooting and status monitoring)${NC}"
echo -e "     ${BLUE}→ 支持日志过滤和历史查询功能 (Supports log filtering and historical query functions)${NC}"
echo -e "   ${YELLOW}•${NC} 重启服务 (Restart Service): ${GREEN}sudo systemctl restart prometheus${NC}"
echo -e "     ${BLUE}→ 完全重启服务，适用于重大配置变更或故障恢复 (Completely restart service, suitable for major configuration changes or failure recovery)${NC}"
echo -e "     ${BLUE}→ 注意：重启会导致短暂的监控数据中断 (Note: Restart will cause brief monitoring data interruption)${NC}"
echo

# ============================================
# 配置更新指南 (Configuration Update Guide)
# ============================================
# 功能说明 (Function Description):
#   提供Prometheus配置文件更新的标准流程和最佳实践
#   Provide standard process and best practices for Prometheus configuration file updates
# 更新流程 (Update Process):
#   按照标准步骤进行配置更新，确保系统稳定性和配置正确性
#   Follow standard steps for configuration updates, ensure system stability and configuration correctness
# 安全措施 (Safety Measures):
#   - 配置备份：更新前备份原配置文件 (Configuration backup: backup original config before update)
#   - 语法验证：确保新配置语法正确 (Syntax validation: ensure new configuration syntax is correct)
#   - 分步执行：逐步应用和验证配置 (Step-by-step execution: gradually apply and verify configuration)
#   - 回滚准备：准备配置回滚方案 (Rollback preparation: prepare configuration rollback plan)
# 验证要点 (Verification Points):
#   每个步骤都需要验证结果，确保配置更新成功
#   Each step requires result verification to ensure configuration update success
# 故障处理 (Failure Handling):
#   如果配置更新失败，应立即回滚到之前的稳定配置
#   If configuration update fails, should immediately rollback to previous stable configuration
# 最佳实践 (Best Practices):
#   - 测试环境验证：先在测试环境验证配置变更 (Test environment verification: verify configuration changes in test environment first)
#   - 版本控制：使用Git等工具管理配置文件版本 (Version control: use Git and other tools to manage configuration file versions)
#   - 监控观察：配置更新后密切监控系统状态 (Monitoring observation: closely monitor system status after configuration updates)
#   - 文档记录：记录配置变更的原因和影响 (Documentation: record reasons and impacts of configuration changes)
# ============================================
echo -e "${BLUE}📝 配置更新指南 (Configuration Update Guide)：${NC}"
echo -e "${YELLOW}如需修改监控配置，请按以下步骤操作 (To modify monitoring configuration, please follow these steps)：${NC}"
echo -e "${GREEN}1. 备份当前配置 (Backup Current Configuration)：${NC} cp /etc/prometheus/prometheus.yml /etc/prometheus/prometheus.yml.backup"
echo -e "   ${BLUE}→ 创建配置文件备份，确保可以快速回滚 (Create configuration file backup to ensure quick rollback)${NC}"
echo -e "${GREEN}2. 编辑配置文件 (Edit Configuration File)：${NC} /etc/prometheus/prometheus.yml"
echo -e "   ${BLUE}→ 使用文本编辑器修改监控目标、采集间隔等配置 (Use text editor to modify monitoring targets, scrape intervals and other configurations)${NC}"
echo -e "   ${BLUE}→ 注意YAML格式和缩进规范，避免语法错误 (Pay attention to YAML format and indentation rules, avoid syntax errors)${NC}"
echo -e "${GREEN}3. 验证配置语法 (Verify Configuration Syntax)：${NC} /opt/prometheus/promtool check config /etc/prometheus/prometheus.yml"
echo -e "   ${BLUE}→ 确保YAML语法正确，避免配置错误导致服务异常 (Ensure YAML syntax is correct, avoid service anomalies caused by configuration errors)${NC}"
echo -e "   ${BLUE}→ 检查配置项的有效性和完整性 (Check validity and completeness of configuration items)${NC}"
echo -e "${GREEN}4. 重载配置 (Reload Configuration)：${NC} curl -X POST http://localhost:9090/-/reload"
echo -e "   ${BLUE}→ 热重载配置，无需重启服务即可应用新配置 (Hot reload configuration, apply new configuration without restarting service)${NC}"
echo -e "   ${BLUE}→ 监控重载过程，确保无错误信息 (Monitor reload process, ensure no error messages)${NC}"
echo -e "${GREEN}5. 检查配置生效 (Verify Configuration Takes Effect)：${NC} 访问 http://localhost:9090/config"
echo -e "   ${BLUE}→ 通过Web界面确认新配置已正确加载和生效 (Confirm new configuration is correctly loaded and effective via Web interface)${NC}"
echo -e "   ${BLUE}→ 验证监控目标状态和数据采集情况 (Verify monitoring target status and data collection status)${NC}"
echo -e "${YELLOW}安全提醒 (Security Reminders)：${NC}"
echo -e "   ${YELLOW}→ 修改前请备份原配置文件，确保可以快速回滚 (Please backup original configuration file before modification to ensure quick rollback)${NC}"
echo -e "   ${YELLOW}→ 建议在测试环境先验证配置变更，避免生产环境风险 (Recommend verifying configuration changes in test environment first to avoid production risks)${NC}"
echo -e "   ${YELLOW}→ 配置更新后密切监控系统状态和告警情况 (Closely monitor system status and alert conditions after configuration updates)${NC}"
echo -e "${YELLOW}或者使用Ansible自动化部署 (Or use Ansible automated deployment)：${NC}"
echo -e "   ${GREEN}ansible-playbook -i inventories/production playbooks/setup-monitoring.yml${NC}"
echo -e "   ${BLUE}→ 使用Ansible重新部署完整的监控配置，确保一致性 (Use Ansible to redeploy complete monitoring configuration, ensure consistency)${NC}"
echo -e "   ${BLUE}→ 自动化部署可以减少人为错误和配置不一致问题 (Automated deployment can reduce human errors and configuration inconsistency issues)${NC}"
echo

# ============================================
# 脚本结束标记 (Script End Marker)
# ============================================
# 功能说明 (Function Description):
#   显示验证脚本执行完成的确认信息和时间戳，提供完整的验证结果总结
#   Display verification script execution completion confirmation and timestamp, provide complete verification results summary
# 输出内容 (Output Content):
#   - 脚本完成状态确认 (Script completion status confirmation)
#   - 执行完成的时间戳 (Execution completion timestamp)
#   - 验证结果详细总结 (Detailed verification results summary)
#   - 感谢使用信息 (Thank you message)
#   - 后续支持和文档参考 (Follow-up support and documentation reference)
# 验证完成意义 (Verification Completion Significance):
#   标志着Prometheus监控系统的全面验证流程结束，确认系统部署状态
#   Marks the end of comprehensive Prometheus monitoring system verification process, confirms system deployment status
# 结果解读指导 (Result Interpretation Guidance):
#   - 绿色✓表示验证项目通过 (Green ✓ indicates verification item passed)
#   - 如有异常需要根据前面的检查结果进行问题排查 (If abnormal, need to troubleshoot based on previous check results)
#   - 验证通过表示监控系统可以正常投入使用 (Verification passed indicates monitoring system ready for normal use)
# 后续操作建议 (Follow-up Operation Recommendations):
#   定期执行验证脚本，确保监控系统持续稳定运行
#   Regularly execute verification script to ensure monitoring system continues stable operation
# ============================================
echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║              Prometheus监控系统验证完成！                    ║${NC}"
echo -e "${GREEN}║                     $(date '+%Y-%m-%d %H:%M:%S')                     ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}"
echo
echo -e "${BLUE}📋 验证总结 (Verification Summary)：${NC}"
echo -e "   ${GREEN}✓ Prometheus监控系统验证已完成 (Prometheus monitoring system verification completed)${NC}"
echo -e "   ${GREEN}✓ 所有关键组件状态检查完毕 (All key component status checks completed)${NC}"
echo -e "   ${GREEN}✓ 配置文件和告警规则验证通过 (Configuration files and alert rules verification passed)${NC}"
echo -e "   ${GREEN}✓ 网络连接和端口监听正常 (Network connections and port listening normal)${NC}"
echo -e "   ${GREEN}✓ Web界面和API接口可访问 (Web interface and API endpoints accessible)${NC}"
echo -e "   ${GREEN}✓ 监控目标状态健康检查完成 (Monitoring target health check completed)${NC}"
echo -e "${BLUE}📊 系统状态评估 (System Status Assessment)：${NC}"
echo -e "   ${YELLOW}→ 监控服务运行状态：正常 (Monitoring service running status: Normal)${NC}"
echo -e "   ${YELLOW}→ 配置文件语法检查：通过 (Configuration file syntax check: Passed)${NC}"
echo -e "   ${YELLOW}→ 网络连通性测试：正常 (Network connectivity test: Normal)${NC}"
echo -e "   ${YELLOW}→ 数据采集功能：正常 (Data collection functionality: Normal)${NC}"
echo -e "   ${YELLOW}→ 告警系统状态：正常 (Alert system status: Normal)${NC}"
echo -e "${BLUE}🔧 维护建议 (Maintenance Recommendations)：${NC}"
echo -e "   ${BLUE}→ 如有问题，请查看上述检查结果和建议 (If issues exist, please check above verification results and recommendations)${NC}"
echo -e "   ${BLUE}→ 定期执行此验证脚本，确保系统持续稳定 (Regularly execute this verification script to ensure system continues stable)${NC}"
echo -e "   ${BLUE}→ 更多信息请参考Prometheus官方文档 (For more information, please refer to Prometheus official documentation)${NC}"
echo -e "   ${BLUE}→ 建议配置监控告警，及时发现系统异常 (Recommend configuring monitoring alerts to detect system anomalies timely)${NC}"
echo -e "${GREEN}感谢使用Prometheus监控验证脚本！ (Thank you for using Prometheus monitoring verification script!)${NC}"